import requests
from bs4 import BeautifulSoup

if __name__ == "__main__":

    #请求的URL
    url = 'https://www.shicimingju.com/book/sanguoyanyi.html'
    #UA伪装
    headers = {
        'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36'
    }
    respones = requests.get(url=url,headers=headers)
    respones.encoding = 'utf-8'
    #构造BeautifulSoup对象
    soup = BeautifulSoup(respones.text,"lxml")
    #解析标签
    #<div class="book-mulu">
    #<a href="/book/sanguoyanyi/1.html">第一回·宴桃园豪杰三结义  斩黄巾英雄首立功</a>
    booklist = soup.select('.book-mulu > ul > li')
    fp = open("sanguo.txt","w",encoding="utf-8")
    for i in booklist:
        #文章标题
        title = i.a.string
        detail_url = 'https://www.shicimingju.com' + i.a['href']
        #请求文章页
        detail_page_request = requests.get(url=detail_url,headers=headers,timeout=5)
        detail_page_request.encoding = 'utf-8'
        detail_page_text = detail_page_request.text
        #解析文章内容
        detial_soup = BeautifulSoup(detail_page_text,'lxml')
        #找到目标div
        div_content = detial_soup.find('div',class_='chapter_content')
        #获取DIV中的文本
        content = div_content.text
        fp.write(title+':'+content+'\n')
        print(title,'爬取成功')
